from langchain_community.document_loaders import TextLoader
print("=== TextLoader 加载结果 ===")
text_loader = TextLoader("D:/ideaSpace/rag-in-action-master/90-文档-Data/灭神纪/人物角色.json", encoding="utf-8")
text_documents = text_loader.load()
print(text_documents)

from langchain_community.document_loaders import JSONLoader
print("=== JSONLoader 加载结果 ===")
print("1. 主角信息：")
main_loader = JSONLoader(
    file_path="D:/ideaSpace/rag-in-action-master/90-文档-Data/灭神纪/人物角色.json",
    jq_schema='.mainCharacter | "姓名：" + .name + "，背景：" + .backstory',
    text_content=True
)
main_char = main_loader.load()
print(main_char)
print("\n2. 支持角色信息：")
support_loader = JSONLoader(
    file_path="D:/ideaSpace/rag-in-action-master/90-文档-Data/灭神纪/人物角色.json",
    jq_schema='.supportCharacters[] | "姓名：" + .name + "，背景：" + .background',
    text_content=True
)
support_chars = support_loader.load()
print(support_chars)


print("------------------------使用WebBaseLoader加载网页-----------------------------")
from langchain_community.document_loaders import WebBaseLoader
page_url = "https://zh.wikipedia.org/wiki/黑神话：悟空"
page_url = "https://blog.csdn.net/weixin_45690643/article/details/136328902"
# 设置用户代理（避免被网站屏蔽）
#os.environ["USER_AGENT"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
loader = WebBaseLoader(web_paths=[page_url])
docs = []
docs = loader.load()
assert len(docs) == 1
doc = docs[0]
print(f"{doc.metadata}\n")
#print(doc.page_content.strip()[:3000])

print("--------------------------------加载Markdown文档-----------------------------------")
from langchain_community.document_loaders import UnstructuredMarkdownLoader

markdown_path = "D:/ideaSpace/rag-in-action-master/90-文档-Data/黑悟空/黑悟空版本介绍.md"
loader = UnstructuredMarkdownLoader(markdown_path)

data = loader.load()
print(data[0].page_content[:250])

loader = UnstructuredMarkdownLoader(markdown_path, mode="elements")
data = loader.load()
print(f"Number of documents: {len(data)}\n")
for document in data:
    print(f"{document}\n")


print("-----------使用unstructured提供的partition_html方法直接解析网页----------------------")
from unstructured.partition.html import partition_html
import requests

url = "https://blog.csdn.net/weixin_45690643/article/details/136328902"
headers = {"User-Agent": "Mozilla/5.0"}  # 模拟浏览器访问
response = requests.get(url, headers=headers)
html_content = response.text

elements = partition_html(text=html_content)  # 解析 HTML
for element in elements[:50]:
    print(element.text)  # 打印解析后的文本


print("-----------使用UnstructuredLoader加载网页（只能处理本地网页）----------------------")
# 执行会报
from langchain_unstructured import UnstructuredLoader
loader = UnstructuredLoader(file_path="D:/tmp/36.云原生之SpringCloud+k8s实践-CSDN博客.html")
docs = loader.load()
for doc in docs[:5]:
    print(f'{doc.metadata["category"]}: {doc.page_content}')